from bs4 import BeautifulSoup
import requests
import pymysql
import time
import logging
from logging.handlers import RotatingFileHandler
class Poetry(object):
    # 年代列表
    __yearsList = [];
    # 诗人列表
    __poetsList = [];
    # 诗词列表
    __poetrysList = [];
    __headers = {
        "Host":"www.shicimingju.com",
        "User-Agent":"Mozilla/5.0(WindowsNT10.0;Win64;x64;rv:58.0)Gecko/20100101Firefox/58.0",
        "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language":"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
        "Accept-Encoding":"gzip,deflate",
        "Cookie":"Hm_lvt_4c1638db937a6ad4a0e6a8bdfa32146f=1518788859,1518789642,1518792869,1518828425;AJSTAT_ok_times=5;Hm_lpvt_4c1638db937a6ad4a0e6a8bdfa32146f=1518834283;AJSTAT_ok_pages=18;__tins__3854170=%7B%22sid%22%3A%201518834051270%2C%20%22vd%22%3A%206%2C%20%22expires%22%3A%201518836083001%7D;__51cke__=;__51laig__=18;top_is_guding=no;shiju=;cate=all;kw=%u522B%u5218%u8ADD;font-weight=%u4E2D;PHPSESSID=upp84qtv7idkj2dpockm88cdc6",
        "Connection":"keep-alive",
        "Upgrade-Insecure-Requests":"1",
        "Cache-Control":"max-age=0"
    }
    __conn = None;
    __cur = None;
    __log = None;

    def __init__(self):
        LOG_FORMAT = "%(asctime)s : %(levelname)s : %(message)s";
        formatter = logging.Formatter(LOG_FORMAT);
        log_file_handler = RotatingFileHandler('my.log', mode='a', maxBytes=1048576);
        log_file_handler.setFormatter(formatter);
        logging.basicConfig(level=logging.DEBUG);
        self.__log = logging.getLogger();
        self.__log.addHandler(log_file_handler);
        self.__conn = conn = pymysql.connect(host='localhost',port = 3306,user='root',passwd='asd123',db ='db',charset='utf8');
        self.__cur = self.__conn.cursor();
        has = False;
        if has != True:
            self.updateYearsList();
        else:
            fo = open('__yearsList.txt','r');
            self.__yearsList = list(eval(fo.read()));
            fo.close();
            fo = open('__poetrysList.txt','r');
            self.__poetrysList = list(eval(fo.read()));
            fo.close();
            fo = open('__poetsList.txt','r')
            self.__poetsList = list(eval(fo.read()));
            fo.close();
    def doSql(self,sql):
        try:
            self.__cur.execute(sql);
            self.__conn.commit();
            self.__log.info('执行sql语句');
        except:
            self.__conn.rollback();
            self.__log.error('数据保存失败！');
        
    # 获取一首诗词链接并在列表中删除，如果诗词列表为空则更新诗词列表
    def getPiceOfPoetry(self):
        if len(self.__poetrysList) > 0:
            poetrys = self.__poetrysList[0];
            del self.__poetrysList[0];
            return poetrys;
        else:
            self.updatePoetrysList();
            return self.getPiceOfPoetry();
    # 获取一个诗人链接并在列表中删除，如果诗人列表为空则更新诗人列表
    def getPiceOfPoet(self):
        if len(self.__poetsList) > 0:
            poets = self.__poetsList[0];
            del self.__poetsList[0];
            return poets;
        else:
            self.updatePoetsList();
            return self.getPiceOfPoet();
    # 获取一个年代链接并在列表中删除，如果年代列表为空则结束程序
    def getPiceOfYears(self):
        if len(self.__yearsList) > 0:
            years = self.__yearsList[0];
            del self.__yearsList[0];
            return years;
        else:
            # print('----------------获取完毕----------------');
            log.info('程序执行完毕！')
            exit();
    # 更新年代列表
    def updateYearsList(self):
        self.__log.info('更新年代列表');
        r = requests.get("http://www.shicimingju.com/",headers = self.__headers);
        r.encoding = 'utf-8';
        soup = BeautifulSoup(r.text,"lxml");
        yearsList = soup.find('div',{'id':'navili_id_3'});
        years = yearsList.find_all('a');
        for year in years:
            self.__yearsList.append('http://www.shicimingju.com' + year['href']);
        # self.__log.debug(self.__yearsList);
    # 更新诗人列表
    def updatePoetsList(self):
        self.__log.info('更新诗人列表');
        yearURL = self.getPiceOfYears();
        fo = open('__yearsList.txt','w+');
        fo.write(str(self.__yearsList));
        fo.flush();
        fo.close();
        r = requests.get(yearURL,headers = self.__headers);
        r.encoding = 'utf-8';
        soup = BeautifulSoup(r.text,"lxml");
        poetsList = soup.find('div',{'class':'shirenlist'});
        poets = poetsList.find_all('a');
        for poet in poets:
            self.__poetsList.append('http://www.shicimingju.com' + poet['href']);
    # 更新诗词列表
    def updatePoetrysList(self):
        self.__log.info('更新诗词列表');
        poetURL = self.getPiceOfPoet();
        # self.__log.debug(self.__poetsList);
        fo = open('__poetsList.txt','w+');
        fo.write(str(self.__poetsList));
        fo.flush();
        fo.close();
        r = requests.get(poetURL,headers = self.__headers);
        r.encoding = 'utf-8';
        soup = BeautifulSoup(r.text,"lxml");
        #  查找下一页链接
        try:
            nextURLs = soup.find('span',{'class':'next'}).find_all('a');
            for nextURL in nextURLs:
                if nextURL.text == '下一页':
                    self.__poetsList.append('http://www.shicimingju.com' + nextURL['href']);
        except:
            self.__log.error('updatePoetrysList.nextURLs : 该人物只有一页诗词数据');
        # 开始获取诗词列表
        poetrys = soup.find('div',{'class':'shicilist'}).find_all('ul');
        # self.__log.debug('updatePoetrysList.poetrys : ' + str(poetrys));
        for poetry in poetrys:
            poetryURL = poetry.find('a');
            self.__poetrysList.append('http://www.shicimingju.com' + poetryURL['href']);
    # 获取诗词详细信息并保存
    def update(self):
        url = self.getPiceOfPoetry();
        self.__log.info('获取诗词信息' + url);
        r = requests.get(url,headers = self.__headers);
        r.encoding = 'utf-8';
        soup = BeautifulSoup(r.text,"lxml");
        poetry = soup.find('div',{'id':'middlediv'}).find('div',{'class':'zhuti yuanjiao'});
        # self.__log.debug('update.poetry : ' + str(poetry));
        sql = 'INSERT INTO db_demo_poetry(info) VALUES("' + pymysql.escape_string(str(poetry)) + '");'
        self.doSql(sql);
        # self.__log.debug(self.__poetrysList);
        fo = open('__poetrysList.txt','w+');
        fo.write(str(self.__poetrysList));
        fo.flush();
        fo.close();
if __name__ == '__main__':
    p = Poetry();
    while True:
        p.update();